import click
from selenium import webdriver
from pathlib import Path
from tqdm import tqdm
from tools import get_current_page, next_page, wait_capcha
import json

@click.command()
@click.option('--urls-file', default='urls.txt')
@click.option('--output-path', default='results')
def main(urls_file, output_path):
    Path(output_path).mkdir(exist_ok=True, parents=True)
    driver = webdriver.Chrome(executable_path='./chromedriver')

    with open(urls_file, 'r') as f:
        urls = f.read().strip().split('\n')

    for i, url in enumerate(tqdm(urls)):
        if url == '0':
            res = ""
        else:
            driver.get(url)
            wait_capcha(driver)
            
            res = ""
            while True:
                res += get_current_page(driver)
                if not next_page(driver):
                    break

        Path(f'{output_path}/{i+1}.txt').write_text(res, encoding="utf-8")
    driver.quit()

if __name__ == "__main__":
    main()
        